# -*- coding:utf-8 -*-
# /usr/bin/env python
"""
Date: 2020/10/21 16:28
Desc: 东方财富网-数据中心-特色数据-千股千评
http://data.eastmoney.com/stockcomment/
"""
import demjson
import pandas as pd
import requests
from tqdm import tqdm

#before 20220516
def stock_em_comment_old() -> pd.DataFrame:
    """
    东方财富网-数据中心-特色数据-千股千评
    http://data.eastmoney.com/stockcomment/
    :return: 千股千评数据
    :rtype: pandas.DataFrame
    """
    url = "http://dcfm.eastmoney.com/em_mutisvcexpandinterface/api/js/get"
    params = {
        "type": "QGQP_LB",
        "token": "70f12f2f4f091e459a279469fe49eca5",
        "cmd": "",
        "st": "Code",
        "sr": "1",
        "p": "1",
        "ps": "10000",
        "js": "var fHdHpFHW={pages:(tp),data:(x),font:(font)}",
        "filter": "",
        "rt": "52831859",
    }
    """
    try:
    
      from http.client import HTTPConnection
    
    except ImportError:
    
      from httplib import HTTPConnection
    
    import logging
    
    HTTPConnection.debuglevel = 1
    
    logging.basicConfig() # 初始化 logging，否则不会看到任何 requests 的输出。
    
    logging.getLogger().setLevel(logging.DEBUG)
    
    requests_log = logging.getLogger("requests.packages.urllib3")
    
    requests_log.setLevel(logging.DEBUG)
    
    requests_log.propagate = True
    """
    r = requests.get(url, params=params)
    data_text = r.text
    data_json = demjson.decode(data_text[data_text.find("{"):])
    data_df = pd.DataFrame(data_json["data"])
    data_df["TDate"] = pd.to_datetime(data_df["TDate"])
    return data_df


#since 20220517
def stock_em_comment() -> pd.DataFrame:
    """
    东方财富网-数据中心-特色数据-千股千评
    http://data.eastmoney.com/stockcomment/
    :return: 千股千评数据
    :rtype: pandas.DataFrame
    ---
    https://datacenter-web.eastmoney.com/api/data/v1/get?
    callback=jQuery112306723101674232936_1653237816438
    &sortColumns=SECURITY_CODE
    &sortTypes=1
    &pageSize=5000
    &pageNumber=1
    &reportName=RPT_DMSK_TS_STOCKNEW
    &quoteColumns=f2~01~SECURITY_CODE~CLOSE_PRICE,f8~01~SECURITY_CODE~TURNOVERRATE,f3~01~SECURITY_CODE~CHANGE_RATE,f9~01~SECURITY_CODE~PE_DYNAMIC
    &columns=ALL
    &filter=
    &token=894050c76af8597a853f5b408b759f5d
    """

    url = "https://datacenter-web.eastmoney.com/api/data/v1/get"
    
    params = {
        #"callback": "jQuery112306723101674232936_165323781643",
        "sortColumns": "SECURITY_CODE",
        "sortTypes":"1",
        "pageSize": "10000",
        "pageNumber": "1",
        "reportName": "RPT_DMSK_TS_STOCKNEW",
        "quoteColumns": "f2~01~SECURITY_CODE~CLOSE_PRICE,f8~01~SECURITY_CODE~TURNOVERRATE,f3~01~SECURITY_CODE~CHANGE_RATE,f9~01~SECURITY_CODE~PE_DYNAMIC",
        "columns": "TRADE_DATE,SECURITY_CODE,SECURITY_NAME_ABBR,CLOSE_PRICE,CHANGE_RATE,PE_DYNAMIC,TURNOVERRATE,PRIME_COST,ORG_PARTICIPATE,PARTICIPATE_TYPE,PRIME_COST_20DAYS,PRIME_COST_60DAYS,SUPERDEAL_INFLOW,SUPERDEAL_OUTFLOW,BIGDEAL_INFLOW,BIGDEAL_OUTFLOW,PRIME_INFLOW,TRADE_MARKET_CODE,TOTALSCORE,RANK_UP,RANK,FOCUS",
        #"columns": "SECURITY_CODE,TRADE_DATE",
        "filter": "",
        "token":"894050c76af8597a853f5b408b759f5d"
    }

    r = requests.get(url, params=params)
    #data_text = r.text
    #data_json = demjson.decode(data_text[data_text.find("{"):])
    #data_json = demjson.decode(data_text)
    data_json = r.json()
    page_num = data_json['result']['pages']
    data_df = pd.DataFrame(data_json["result"]["data"])
    for page in tqdm(range(1, page_num + 1)):
        params.update({'pageNumber': page})
        r = requests.get(url, params=params)
        data_json = r.json()
        temp_df = pd.DataFrame(data_json['result']["data"])
        data_df = data_df.append(temp_df, ignore_index=True)

    data_df["TRADE_DATE"] = pd.to_datetime(data_df["TRADE_DATE"])
    return data_df



if __name__ == '__main__':
    stock_em_comment_df = stock_em_comment()
    print(stock_em_comment_df)
